package com.gxljc.bear.crawler.base;

import com.summba.crawl.parse.AbstractHtmlParser;
import com.gxljc.commons.config.ConfigUtil;
import com.gxljc.commons.util.Pair;
import com.gxljc.commons.util.StringUtil;
import com.gxljc.commons.util.config.CommConsts;
import com.gxljc.bear.crawler.proxy.IpObject;
import com.gxljc.bear.crawler.proxy.ProxyManager;
import com.gxljc.bear.crawler.util.HtmlUtil;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.log4j.Logger;
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.parse.Parse;
import org.apache.nutch.parse.ParseResult;
import org.apache.nutch.protocol.Content;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class DolphinFetchData implements Serializable {
    private static final long serialVersionUID = 20L;
    private static DolphinFetchData fd = null;
    private static Logger LOG = Logger.getLogger(DolphinFetchData.class);

    private static Configuration conf = HBaseConfiguration.create();
    private static List<AbstractHtmlParser> parsers = null;

    private static final String[] DEFAULT_PARAM_CONFIGS = new String[]{CommConsts.SERV_CRAWLER_ROOT_DIR};
    private static final String[] DEFAULT_SIMPLE_SERV_CONFIGS = new String[]{};
    // private static final int DEFAULT_MAX_READ_BYTE = 1024 * 1024 * 2; // 2MB

    private static final String DEFAULT_TEMPLATE_DIR = "template";
    private static final String DEFAULT_PARSER = "com.summba.crawl.parse.general.TemplateHtmlParser";

    //	private static final String USER_AGENT = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)";
    private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.94 Safari/537.36";
    private String cacheCookie = "bid=\"7Hku0QjxH11\"";
    private String BAIDU_ZHIDAO_COOKIE_FORMAT = "bdshare_firstime=1432633892229; IK_CID_1101=2; IK_CID_79=10; BAIDUID=6577F45ED4DDB6D16B2B2DB9D5F68F04:FG=1; PSTM=1458184246; BIDUPSID=5109ADBB142F20D60E3E282C15455D93; IK_CID_95=18; IK_CID_78=14; IK_CID_1031=9; IK_CID_84=40; IK_CID_1=3; IK_CID_83=38; IK_CID_82=13; BDUSS=mlVbndJbVNiS2E0S2NZYm9vbjZRMThifkJaLTZXY2VJSjRrVC1vd3lHbTczaGxYQVFBQUFBJCQAAAAAAAAAAAEAAABu6pR-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALtR8la7UfJWZ; IK_USERVIEW=1; IK_CID_80=22; IK_CID_77=26; IK_CID_85=25; Hm_lvt_6859ce5aaf00fb00387e6434e4fcc925=1458720217; Hm_lpvt_6859ce5aaf00fb00387e6434e4fcc925=%d; IK_6577F45ED4DDB6D16B2B2DB9D5F68F04=17; IK_CID_81=18; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; H_PS_PSSID=18880_1444_18205_17000_14977_11940_19369_16915";

    private final String PROXY_URL = "http://%s:8081/getContent"; //代理server URL
    private final String PROXY_SERVER_USER = "newproxyuserid";

    /**
     * 根据不同的频道构造FetchData
     *
     * @param type
     * @throws Exception
     */
    private DolphinFetchData(String type) throws Exception {
        if (parsers == null) {
            parsers = new ArrayList<AbstractHtmlParser>();

            conf.addResource(DolphinCrawlerConsts.FILE_DOLPHIN_CRAWLER);
            ConfigUtil.fastLoadConfig(conf, CommConsts.ORG, CommConsts.APP,
                    DEFAULT_PARAM_CONFIGS, DEFAULT_SIMPLE_SERV_CONFIGS);

            setXpathTemplate(type);
            String[] parserArr = DEFAULT_PARSER.split(",");
            for (String parserStr : parserArr) {
                Class<?> cl = null;
                try {
                    cl = Class.forName(parserStr);
                } catch (ClassNotFoundException e) {
                    e.printStackTrace();
                }
                if (cl != null) {
                    AbstractHtmlParser parser = (AbstractHtmlParser) cl
                            .newInstance();
                    parser.setConf(conf);
                    parsers.add(parser);
                }
            }
            ProxyManager.getInstance().init(conf);
        }
    }

    private void setXpathTemplate(String type) {
        String servRootDir = conf.get(CommConsts.SERV_CRAWLER_ROOT_DIR);
        String tmpDirParent = conf.get("src.parser.xpath.template.type");
        String[] array = tmpDirParent.split(StringUtil.STR_DELIMIT_1ST);
        String tmpDir = null;
        for (String str : array) {
            String[] item = str.split(StringUtil.STR_DELIMIT_3RD);
            if (type.equals(item[0]))
                tmpDir = item[1].trim();
        }
        if (StringUtil.isNullOrEmpty(tmpDir))
            tmpDir = DEFAULT_TEMPLATE_DIR;
        if (servRootDir.endsWith("/") && tmpDir.startsWith("/"))
            tmpDir = tmpDir.substring(1);
        conf.set("parser.xpath.template",
                FilenameUtils.concat(servRootDir, tmpDir));
    }

    public static DolphinFetchData getInstance(String type) throws Exception {
        if (fd == null) {
            synchronized (DolphinFetchData.class) {
                if (fd == null)
                    fd = new DolphinFetchData(type);
            }
        }
        return fd;
    }

    private String getPage(String url, Proxy proxy) throws IOException {
        cacheCookie = genRandomBrowserId();
        GetMethod getMethod = genGetMethod(url, proxy);
        String html = getMethod.getResponseBodyAsString();
        getMethod.abort();

        if (isForbid(html)) {
            LOG.error("forbidden!!!!");
            // System.out.println(html);
            String bid = "frJCIQ1GepQ"
                    + Integer.toString(new Random().nextInt(9))
                    + Integer.toString(new Random().nextInt(9));
            cacheCookie = "bid=\"" + bid + "\"";
            getMethod = genGetMethod(url, proxy);
            html = getMethod.getResponseBodyAsString();
            String browserId = getFirstBetween(html, "browserId = '", "'");
            cacheCookie = String.format("bid=\"%s\";ue=\"%s\"", browserId,
                    "bill_01@163.com");
            getMethod.abort();
//			System.out.println(html);

        }
        return html;
    }

    private GetMethod genGetMethod(String url, Proxy proxy)
            throws HttpException, IOException {
        HttpClient httpClient = new HttpClient();
        GetMethod getMethod = new GetMethod(url);
        getMethod.setRequestHeader("User-Agent", USER_AGENT);
        getMethod.setRequestHeader("Cookie", cacheCookie);
        httpClient.getHttpConnectionManager().getParams()
                .setConnectionTimeout(5000);
        httpClient.getHttpConnectionManager().getParams().setSoTimeout(20000);
        if (proxy != null) {
            InetSocketAddress address = (InetSocketAddress) proxy.address();
            String proxyHost = address.getHostName();
            int proxyPort = address.getPort();
            httpClient.getHostConfiguration().setProxy(proxyHost, proxyPort);
        }
        httpClient.executeMethod(getMethod);
        return getMethod;
    }

    private HttpURLConnection setProperty(HttpURLConnection conn, int timeout) {
        return setProperty(conn, timeout, null);
    }

    private HttpURLConnection setProperty(HttpURLConnection conn, int timeout, Map<String, String> property) {
        conn = setPropertyTimeOut(conn, timeout);
        conn.setRequestProperty("User-Agent", USER_AGENT);
        conn.setRequestProperty("Connection", "keep-alive");
        conn.setRequestProperty("Cache-Control", "max-age=0");
        conn.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.8");
        conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        conn.setRequestProperty("Pragma", "no-cache");
        if (property != null && !property.isEmpty()) {
            for (Map.Entry entry : property.entrySet()) {
                String key = (String) entry.getKey();
                String value = (String) entry.getValue();
                conn.setRequestProperty(key, value);
            }
        }
        return conn;
    }

    private HttpURLConnection setPropertyTimeOut(HttpURLConnection conn, int timeout) {
        conn.setConnectTimeout(timeout);
        conn.setReadTimeout(timeout);
        return conn;
    }

    private HttpURLConnection getConnectionWithRetry(URL url, boolean useProxy) {
        return getConnectionWithRetry(url, useProxy, 20000, null);
    }

    private HttpURLConnection getConnectionWithRetry(URL url, boolean useProxy, int timeout, Map<String, String> property) {
        int retry = conf.getInt("connection.try.time", 3);
        while (retry-- > 0) {
            try {
                HttpURLConnection conn = null;
                if (useProxy) {
                    ProxyManager proxyManager = ProxyManager.getInstance();
                    Proxy proxy = proxyManager.getProxy();
                    System.out.println(proxy);
                    LOG.error(proxy);
                    conn = (HttpURLConnection) url.openConnection(proxy);
                } else {
                    conn = (HttpURLConnection) url.openConnection();
                }
                conn = setProperty(conn, timeout, property);
                conn.connect();
                int connectionResponse = conn.getResponseCode();
                if (connectionResponse == 200) {
                    return conn;
                } else {
                    LOG.error(String.format("ResponseCode=%d, retry=%d",
                            connectionResponse, retry));
                }
            } catch (Exception e) {
                LOG.error("getConnectionWithRetry=" + url, e);
                continue;
            }
        }
        return null;

    }


    /**
     * 搜索
     * url: url
     * decode:url字符集
     */
    public String getProxyServer2Result(String url, String decode, int timeOut) {
        HttpURLConnection conn = null;
        byte[] bytes = null;
        InputStream in = null;
        try {
            ProxyManager proxyManager = ProxyManager.getInstance();
            IpObject ipObject = proxyManager.getIpObject();
            String proxyServerUrl = String.format(PROXY_URL, ipObject.getHost());
//            String proxyServerUrl = String.format(PROXY_URL, "182.147.203.168");
            URL Url = new URL(proxyServerUrl);
            conn = (HttpURLConnection) Url.openConnection();
            conn = setProperty(conn, timeOut);
            conn.setRequestMethod("POST");// 提交模式
            conn.setDoOutput(true);// 是否输入参数

            StringBuffer params = new StringBuffer();
            params.append("id").append("=").append(PROXY_SERVER_USER).append("&")
                    .append("url").append("=").append(url).append("&")
                    .append("decode").append("=").append(decode);
            byte[] urlParams = params.toString().getBytes();
            conn.getOutputStream().write(urlParams);// 输入参数
            conn.connect();
            int connectionResponse = conn.getResponseCode();
            if (connectionResponse != 200) {
                return null;
            }
            in = conn.getInputStream();
            bytes = IOUtils.toByteArray(in);
            if (bytes == null || bytes.length <= 0) {
                return null;
            } else {
                String check = new String(bytes, decode);
                if (check.length() < 2000) return null;
            }
            return new String(bytes, decode);
        } catch (Exception e) {
            LOG.error("errorURL=" + url, e);
        } finally {
            if (in != null) {
                try {
                    in.close();
                } catch (IOException e) {
                    LOG.error("=== In finally block", e);
                }
            }
            if (conn != null) {
                conn.disconnect();
            }
        }
        return null;
    }

    private HttpURLConnection getConnectionWithRetryLocation(URL url, boolean useProxy, int timeout, Map<String, String> property) {
        int retry = conf.getInt("connection.try.time", 3);
        while (retry-- > 0) {
            try {
                HttpURLConnection conn = null;
                if (useProxy) {
                    ProxyManager proxyManager = ProxyManager.getInstance();
                    Proxy proxy = proxyManager.getProxy();
                    System.out.println(proxy);
                    conn = (HttpURLConnection) url.openConnection(proxy);
                } else {
                    conn = (HttpURLConnection) url.openConnection();
                }
                conn = setProperty(conn, timeout, property);
                conn.connect();
                String location = conn.getHeaderField("Location");
                if (!StringUtil.isNullOrEmpty(location)) {
                    URL serverUrl = new URL(location);
                    if (useProxy) {
                        ProxyManager proxyManager = ProxyManager.getInstance();
                        Proxy proxy = proxyManager.getProxy();
                        conn = (HttpURLConnection) serverUrl.openConnection(proxy);
                    } else {
                        conn = (HttpURLConnection) serverUrl.openConnection();
                    }
                }
                int connectionResponse = conn.getResponseCode();
                if (connectionResponse == 200) {
                    return conn;
                } else {
                    LOG.error(String.format("ResponseCode=%d, retry=%d",
                            connectionResponse, retry));
                }
            } catch (Exception e) {
                LOG.error("getConnectionWithRetry=" + url, e);
                continue;
            }
        }
        return null;
    }

    public String getStreamPage(byte[] bytes, String url)
            throws HttpException, IOException {
        Configuration conf = new Configuration();
        String html = new String(bytes);
        final String patternUrl = "<meta[ ]+charset=['\"]?(.*?)['\"]?[ ]?[/]?>";
        Pattern p = Pattern.compile(patternUrl);
        Matcher m = p.matcher(html);
        String defaultCharset = "GBK";
        if (m.find()) {
            defaultCharset = m.group(1);
        }
        String encoding = AbstractHtmlParser.guessEncoding(new Content(url,
                url, bytes, "text/html", new Metadata(), conf), conf, defaultCharset);
        System.out.println("encoding=" + encoding);
        return new String(bytes, encoding);
    }

    public String getHtmlCharset(byte[] bytes) {
        String html = new String(bytes);
//		System.out.println(html);
        String patternUrl1 = "<(?i)meta[ ]+charset=['\"]?(.*?)['\"]?[ ]?[/]?>";
        String patternUrl2 = "<(?i)meta[ ]+name=['\"]?charset['\"]?[ ]+content=['\"]?(.*?)['\"]?[/]?>";
        String patternUrl3 = "<(?i)meta.*?http-equiv=['\"]?Content-Type.*?charset=(.*?)['\"]?[ ]?[/]?>";
        String patternUrl4 = "<(?i)meta[ ]+content=.*?charset=(.*?)['\"]?[ ]+http-equiv=.*?>";
        String[] array = new String[]{patternUrl1, patternUrl2, patternUrl3, patternUrl4};
        String defaultCharset = "GBK";
        for (String pu : array) {
            Pattern p = Pattern.compile(pu);
            Matcher m = p.matcher(html);
            if (m.find()) {
                return m.group(1);
            }
        }
        return defaultCharset;
    }

    public byte[] getHtml(String url, boolean useProxy, int timeout) {
        return getHtml(url, useProxy, timeout, null);
    }

    public byte[] getHtml(String url, boolean useProxy, int timeout, Map<String, String> property) {
        InputStream in = null;
        HttpURLConnection conn = null;
        try {
            URL Url = new URL(url);
            conn = getConnectionWithRetry(Url, useProxy, timeout, property);
            if (conn == null) {
                return null;
            }
            in = conn.getInputStream();
            byte[] bytes = IOUtils.toByteArray(in);
            in.close();
            in = null;
            conn.disconnect();
            conn = null;
            return bytes;
        } catch (Exception e) {
            LOG.error("errorURL=" + url, e);
        } finally {
            if (in != null) {
                try {
                    in.close();
                } catch (IOException e) {
                    LOG.error("=== In finally block", e);
                }
            }
            if (conn != null) {
                conn.disconnect();
            }
        }
        return null;
    }

    //获取已编码后的html
    public String getEncodeHtml(String url, boolean useProxy, int timeout) {
        return getEncodeHtml(url, useProxy, timeout, null);
    }

    //获取已编码后的html
    public String getEncodeHtml(String url, boolean useProxy, int timeout, Map<String, String> property) {
        InputStream in = null;
        HttpURLConnection conn = null;
        BufferedReader inReader = null;
        try {
            URL Url = new URL(url);
            conn = getConnectionWithRetryLocation(Url, useProxy, timeout, property);
            if (conn == null) {
                return null;
            }
            in = conn.getInputStream();
            byte[] bytes = IOUtils.toByteArray(in);
            in.close();
            in = null;
            conn.disconnect();
            conn = null;
            String charset = getHtmlCharset(bytes);
            return new String(bytes, charset);
        } catch (Exception e) {
            LOG.error("errorURL=" + url, e);
        } finally {
            if (in != null) {
                try {
                    in.close();
                } catch (IOException e) {
                    LOG.error("=== In finally block", e);
                }
            }
            if (inReader != null) {
                try {
                    inReader.close();
                } catch (IOException e) {
                    LOG.error("=== In finally block", e);
                }
            }
            if (conn != null) {
                conn.disconnect();
            }
        }
        return null;
    }

    public byte[] getWeixinHtml(String url, String firstUrl, boolean useProxy, int timeout, String cookie) {
        InputStream in = null;
        HttpURLConnection conn = null;
        try {
            URL Url = new URL(firstUrl);
            int retry = conf.getInt("connection.try.time", 3);
            ProxyManager proxyManager = ProxyManager.getInstance();
            Proxy proxy = null;
            while (retry-- > 0) {
                try {
                    conn = null;
                    if (useProxy) {
                        proxy = proxyManager.getProxy();
                        System.out.println(proxy.toString());
                        conn = (HttpURLConnection) Url.openConnection(proxy);
                    } else {
                        conn = (HttpURLConnection) Url.openConnection();
                    }
                    conn = setPropertyTimeOut(conn, timeout);
                    conn.setRequestProperty("User-Agent", USER_AGENT);
                    conn.setRequestProperty("Connection", "keep-alive");
                    conn.setRequestProperty("Cache-Control", "max-age=0");
                    conn.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.8");
                    conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
                    conn.setRequestProperty("Pragma", "no-cache");
                    conn.setRequestProperty("Referer", "http://weixin.sogou.com/");
                    conn.setRequestProperty("Cookie", cookie);
                    conn.connect();
                    int connectionResponse = conn.getResponseCode();
                    if (connectionResponse == 200) {
                        break;
                    } else {
                        LOG.error(String.format("ResponseCode=%d, retry=%d",
                                connectionResponse, retry));
                    }
                } catch (Exception e) {
                    LOG.error("getConnectionWithRetry=" + url, e);
                    continue;
                }
            }
            if (conn == null) {
                return null;
            }
            URL Url2 = new URL(url);
            retry = conf.getInt("connection.try.time", 3);
            while (retry-- > 0) {
                try {
                    if (useProxy) {
                        conn = (HttpURLConnection) Url2.openConnection(proxy);
                    } else {
                        conn = (HttpURLConnection) Url2.openConnection();
                    }
//					conn = (HttpURLConnection) Url2.openConnection();
                    conn = setPropertyTimeOut(conn, timeout);
                    conn.setRequestProperty("User-Agent", USER_AGENT);
                    conn.setRequestProperty("Connection", "keep-alive");
                    conn.setRequestProperty("Cache-Control", "max-age=0");
                    conn.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.8");
                    conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
                    conn.setRequestProperty("Pragma", "no-cache");
                    conn.setRequestProperty("Referer", "http://weixin.sogou.com/");
                    conn.setRequestProperty("Cookie", cookie);
                    conn.connect();
                    int connectionResponse = conn.getResponseCode();
                    if (connectionResponse == 200) {
                        break;
                    } else {
                        LOG.error(String.format("ResponseCode=%d, retry=%d",
                                connectionResponse, retry));
                    }
                } catch (Exception e) {
                    LOG.error("getConnectionWithRetry=" + url, e);
                    continue;
                }
            }
            in = conn.getInputStream();
            byte[] bytes = IOUtils.toByteArray(in);
            in.close();
            in = null;
            conn.disconnect();
            conn = null;
            return bytes;
        } catch (Exception e) {
            LOG.error("errorURL=" + url, e);
        } finally {
            if (in != null) {
                try {
                    in.close();
                } catch (IOException e) {
                    LOG.error("=== In finally block", e);
                }
            }
            if (conn != null) {
                conn.disconnect();
            }
        }
        return null;
    }

    //登录后访问页面
    public byte[] loginAndAccess(String loginUrl, String dataUrl, int timeOut) {
        return loginAndAccess(loginUrl, dataUrl, timeOut, false);
    }

    //登录后访问页面
    public byte[] loginAndAccess(String loginUrl, String dataUrl, int timeOut, boolean useProxy) {
        int retry = conf.getInt("connection.try.time", 3);
        byte[] bytes = null;
        while (retry-- > 0) {
            try {
                HttpClient httpClient = new HttpClient();
                httpClient.getHttpConnectionManager().getParams()
                        .setConnectionTimeout(timeOut);
                httpClient.getHttpConnectionManager().getParams().setSoTimeout(timeOut);
                // 模拟登陆，按实际服务器端要求选用 Post 或 Get 请求方式
                GetMethod postMethod = new GetMethod(loginUrl);
                postMethod.addRequestHeader("Content-Type", "application/x-www-form-urlencoded;charset=utf-8");
                postMethod.addRequestHeader("Connection", "close");
                if (useProxy) {
                    addProxy(httpClient);
                }
                try {
                    // 设置 HttpClient 接收 Cookie,用与浏览器一样的策略
//            httpClient.getParams().setCookiePolicy(
//                    CookiePolicy.BROWSER_COMPATIBILITY);
                    httpClient.executeMethod(postMethod);
                    GetMethod getMethod = new GetMethod(dataUrl);
                    httpClient.executeMethod(getMethod);
                    // 打印出返回数据，检验一下是否成功
                    InputStream stream = getMethod.getResponseBodyAsStream();
                    bytes = IOUtils.toByteArray(stream);
                    getMethod.abort();
                    postMethod.abort();
                    return bytes;
                } catch (Exception e) {
                    e.printStackTrace();
                    return null;
                }
            } catch (Exception er) {
                er.printStackTrace();
                continue;
            }
        }
        return bytes;
    }

    //追加代理功能
    private void addProxy(HttpClient httpClient) {
        ProxyManager proxyManager = ProxyManager.getInstance();
        Proxy proxy = proxyManager.getProxy();
        LOG.error(proxy);
        InetSocketAddress address = (InetSocketAddress) proxy.address();
        String proxyHost = address.getHostName();
        int proxyPort = address.getPort();
        httpClient.getHostConfiguration().setProxy(proxyHost, proxyPort);
    }

    public byte[] getHtml(String url, boolean useProxy) {
        return getHtml(url, useProxy, 20000);
    }

    /**
     * 使用代理搜索百度
     * url: url
     * useProxy:是否使用代理
     * getCookie：是否要获取cookie
     * type:zhidao,baidu
     */
    public Pair<String, byte[]> getResult2Pair(String url, boolean useProxy, int timeout) throws Exception {
        return getResult2Pair(url, useProxy, timeout, null);
    }

    public Pair<String, byte[]> getResult2Pair(String url, boolean useProxy, int timeout, Map<String, String> property) throws Exception {
        HttpURLConnection conn = null;
        byte[] bytes = null;
        InputStream in = null;
        try {
            URL Url = new URL(url);
            String host = HtmlUtil.getHeadHost4URL(url);
            int retry = 3;
            while (retry-- > 0) {
                try {
                    conn = null;
                    Proxy proxy = null;
                    if (useProxy) {
                        ProxyManager proxyManager = ProxyManager.getInstance();
                        proxy = proxyManager.getProxy();
                        conn = (HttpURLConnection) Url.openConnection(proxy);
                    } else {
                        conn = (HttpURLConnection) Url.openConnection();
                    }
                    conn = setProperty(conn, timeout, property);
                    conn.setRequestProperty("Host", host);
                    conn.setRequestProperty("Referer", url);
                    conn.connect();
                    int connectionResponse = conn.getResponseCode();
                    if (connectionResponse != 200) {
                        LOG.error(String.format("ResponseCode=%d, retry=%d",
                                connectionResponse, retry));
                        continue;
                    }
                    in = conn.getInputStream();
                    bytes = IOUtils.toByteArray(in);
                    if (bytes == null || bytes.length <= 0) {
                        continue;
                    } else {
                        String check = new String(bytes);
                        if (check.length() > 2000) break;
                    }
                } catch (Exception e) {
                    LOG.error("getConnectionWithRetry=" + url, e);
                    continue;
                } finally {
                    if (in != null) {
                        in.close();
                    }
                    in = null;
                    if (conn != null) {
                        conn.disconnect();
                    }
                    conn = null;
                    if (bytes == null) continue;
                }
            }
            for (AbstractHtmlParser parser : parsers) {
                ParseResult pr = parser.getParse(new Content(url, url, bytes,
                        "text/html", new Metadata(), conf));
                if (pr != null) {
                    Parse p = pr.get(url);
                    if (p != null) {
                        return new Pair<String, byte[]>(p.getText(), bytes);
                    }
                }
            }
            LOG.error("Parse null url= " + url);
        } catch (Exception e) {
            LOG.error("errorURL=" + url, e);
        } finally {
            if (in != null) {
                try {
                    in.close();
                } catch (IOException e) {
                    LOG.error("=== In finally block", e);
                }
            }
            if (conn != null) {
                conn.disconnect();
            }
        }
        return null;
    }

    public String getResult(String url, boolean useProxy) {
        return getResult(url, useProxy, 20000);
    }

    public String getResult(String url, boolean useProxy, int timeout) {
        return getResult(url, useProxy, timeout, null);
    }

    public String getResult(String url, boolean useProxy, int timeout, Map<String, String> property) {
        InputStream in = null;
        HttpURLConnection conn = null;
        try {
            URL Url = new URL(url);
            conn = getConnectionWithRetryLocation(Url, useProxy, timeout, property);
            if (conn == null) {
                return null;
            }
            in = conn.getInputStream();
            byte[] bytes = IOUtils.toByteArray(in);
            in.close();
            in = null;
            conn.disconnect();
            conn = null;
            for (AbstractHtmlParser parser : parsers) {
                ParseResult pr = parser.getParse(new Content(url, url, bytes,
                        "text/html", new Metadata(), conf));
                if (pr != null) {
                    Parse p = pr.get(url);
                    if (p != null)
                        return p.getText();
                }
            }
            LOG.error("Parse null url= " + url);
        } catch (Exception e) {
            LOG.error("errorURL=" + url, e);
        } finally {
            if (in != null) {
                try {
                    in.close();
                } catch (IOException e) {
                    LOG.error("=== In finally block", e);
                }
            }
            if (conn != null) {
                conn.disconnect();
            }
        }
        return null;
    }

    public String parserResult(String url, String html) {
        try {
            for (AbstractHtmlParser parser : parsers) {
                ParseResult pr = parser.getParse(new Content(url, url, html.getBytes(),
                        "text/html", new Metadata(), conf));
                if (pr != null) {
                    Parse p = pr.get(url);
                    if (p != null)
                        return p.getText();
                }
            }
            LOG.error("Parse null url= " + url);
        } catch (Exception e) {
            LOG.error("errorURL=" + url, e);
        }
        return null;
    }

    private boolean isForbid(String html) {
        // if (html.contains("403 Forbidden") || !html.contains("browserId")) {
        if (html.contains("403 Forbidden")) {
            return true;
        } else {
            return false;
        }
    }

    public static String getFirstBetween(String content, String before,
                                         String after) {
        try {
            int i = content.indexOf(before);
            if (i < 0)
                return null;
            content = content.substring(i + before.length());
            i = content.indexOf(after);
            if (i < 0)
                return null;
            return content.substring(0, i);
        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }
    }

    private String genRandomBrowserId() {
        return String.format("bid=\"%s\"", randomString(11));
    }

    private final static Random randGen = new Random();
    private static char[] numbersAndLetters = null;

    static {
        synchronized (randGen) {
            if (numbersAndLetters == null) {
                synchronized (randGen) {
                    numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxgxljc"
                            + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXgxljc")
                            .toCharArray();
                }
            }
        }
    }

    private static final String randomString(int length) {
        if (length < 1) {
            return null;
        }
        char[] randBuffer = new char[length];
        for (int i = 0; i < randBuffer.length; i++) {
            randBuffer[i] = numbersAndLetters[randGen.nextInt(71)];
        }
        return new String(randBuffer);
    }
}
